Query product price into CSV files
Below is an example of web scraping of price into csv files using Python. Not the best, but it does the job by using requests and beautifulsoup modules.
there are other modules out there to do the same tasks, i.e Selenium and Scrapy, < – still learning them.
import requests
import csv
from bs4 import BeautifulSoup
#initialize csv file writer.
f = csv.writer(open(‘courtsprice.csv’, ‘w’, newline=”))
f.writerow([‘Page’,’Product’, ‘Price1′,’Price2’])
#url that we gonna query.
domain=’https://www.courts.com.sg/campaigns/stay-home-must-haves’
pages=[]
for i in range(5):
url = (domain + ‘?p=’ + str(i) )
pages.append(url)
print(url)
for item in pages:
page = requests.get(item)
soup = BeautifulSoup(page.content, ‘html.parser’)
product_name_list = soup.find(class_=’products list items product-items’)
product_name_list_items = product_name_list.find_all(‘li’, class_=’item product product-item’)
for eachproduct_item in product_name_list_items:
product_name = eachproduct_item.find(‘a’, class_=’product-item-link’)
price_elem = eachproduct_item.find(‘div’, class_=’price-box price-final_price’)
prices= price_elem.find_all(‘span’, class_=’price-wrapper’)
price=”
try:price1=(prices[0].get(‘data-price-amount’))
except IndexError:
price1=”
try:price2=(prices[1].get(‘data-price-amount’))
except IndexError:
price2=”
# write to CSV files
f.writerow([item,product_name.text.strip(),price1,price2])